#! /usr/bin/env python
#coding=utf-8

"""
  2011 Boris Tatarintsev

  Contains API for
  1. Easily login to any LJ account
  2. Get friends list for user
  3. Get user posts

"""

import urllib
import urllib2
import hashlib
import re
import copy
import time
import threading

from funclib import *
from hashtree import *

# exceptions types

class ServerError(BaseException):
	pass

class AuthError(BaseException):
	pass

class ParseError(BaseException):
	pass

class SocketError(BaseException):
    pass

class LJUserSession:    
    
    # regexp to extract friends references HTML code

    reFetchFriends = re.compile("([PC])\>\s(.*)")
    reWhoFriendedMe = re.compile("[PC]\<\s(.*)")
    reFetchInterests = re.compile("\d+?\s\d+?\s(.*)")
    reInvalidUser = re.compile("\!\s(invalid user).*")

    TIMEOUT_SEC = 10

    def __init__(self, username, password):
        self.flatServerUrl = u"http://livejournal.com/interface/flat"
        self.authServerUrl = u"http://www.livejournal.com/login.bml"
        self.profileUrl = u"http://www.livejournal.com/profile.bml"
        
        self.m_stat = StatDataStruct()
        
        self.m_userName = username
        self.m_password = password
    
        self.m_skipCommunities = True        
        self.m_verbose = False
        self.m_collectInterests = False
        self.m_scanMutualFriendsOnly = False
        self.m_updateCallback = None

    def __getChalResponse(self):
        # get authorization response from the server
        dataDict = {"mode": "getchallenge"}
        data = urllib.urlencode(dataDict)
        request = urllib2.Request(self.flatServerUrl, data)
        
        try:
            fp = urllib2.urlopen(request, timeout = LJUserSession.TIMEOUT_SEC)
        except:
            raise SocketError
            
        text = fp.read()        
        fp.close()        
        return self.__parseAuthResponse(text)

    def __parseAuthResponse(self, textResponse):
        # parses authorization response into hash
        elements = textResponse.split()
        # split to groups by two elements
        groups = zip(*[iter(elements)] * 2)
        # return dict
        return dict(groups)
    
    def __getAuthResponse(self, password, challenge):
        # if I need auth some day :)
    	hpass = hashlib.md5(password).hexdigest()
    	response = hashlib.md5(challenge + hpass).hexdigest()
    	return response
    
    def __openUrlWithCookie(self, url, cookiejar):
    	cookieHandler = urllib2.HTTPCookieProcessor(cookiejar)
    	opener = urllib2.build_opener(cookieHandler)	    
    	request = urllib2.Request(url)    
    	fp = opener.open(request)
    	text = fp.read()
    	fp.close()    
    	return text
    
    def __buildUserUrl(self, comm = False, username = ""):
        name = self.m_userName
        if username != "":
            name = username
        # setup community flag
        commFlag = '1'
        if not comm:
            commFlag = '0'
        return 'http://www.livejournal.com/misc/fdata.bml?comm=' + commFlag + '&user=' + name
    
    def __buildUserInterestUrl(self, username = ""):
        name = self.m_userName
        if username != "":
            name = username
        return 'http://www.livejournal.com/misc/interestdata.bml?user=' + name
    
    def __decodeToUtf8(self, text):
        try:
            # try to decode to utf-8
            output = text.decode('utf-8')
        except UnicodeDecodeError:
            # if we can't decode the string just use it as it is
            output = text
        return output
    
    def __addAndLink(self, parent, child, communityFlag):
        # creates a child node and links parent with the newly created child
        if not self.m_friendsTree.hasNode(child):
            self.m_friendsTree.addNode(child)
            self.m_friendsTree.addNodeData(child, 'C', communityFlag)
            self.m_friendsTree.setRef(parent, child)
        
    def __setNodeDepth(self, nodeName, depth):
        # we sort all the nodes on graph starting from root so we only
        # change the depth value if it is smaller than the previous one (closer to the root)
        if self.m_friendsTree.getDepth(nodeName) == None or self.m_friendsTree.getDepth(nodeName) > depth:
            self.m_friendsTree.setDepth(nodeName, depth)
 
    def __getUserFriends(self, comm = False, username = ""):
        # returns the list of the given LJ user friends        
        urlToOpen = self.__buildUserUrl(comm, username)
        request = urllib2.Request(urlToOpen)
        try:
            fp = urllib2.urlopen(request, timeout = LJUserSession.TIMEOUT_SEC)
            text = fp.read()
            fp.close()
            return text
        except:
            self.m_stat.m_totalNetErrSkip += 1
            return None
            #raise SocketError
            
    def __getUserInterests(self, username):
        #if self.m_collectInterests:
        # fetch user interests 
        urlToOpen = self.__buildUserInterestUrl(username)        
        request = urllib2.Request(urlToOpen)
        try:
           fp = urllib2.urlopen(request, timeout = LJUserSession.TIMEOUT_SEC)
           text = fp.read()
           fp.close()
           return text
        except:
           self.m_stat.m_totalNetErrSkip += 1
           return None
           #raise SocketError
        
    def __fetchUserInterestsFromData(self, text):
        # parse fetched data and extract user interests
        if text != None:
            m2 = LJUserSession.reFetchInterests.findall(text)
            if m2:
                output = [0] * len(m2)
                idx = 0
                # reduce URL data and add communities flag
                for item in m2:
                    output[idx] = self.__decodeToUtf8(item)
                    idx += 1                
                return output
        return None
    
    def __fetchUserFriendsFromData(self, text):
        # parse fetched data and extract friends
        # returns friend as an list of tuples
        # each tuple has link to friend's profile as its first element and
        # the friend's name as a second one
        if text != None:
            m1 = LJUserSession.reWhoFriendedMe.findall(text)
            m2 = LJUserSession.reFetchFriends.findall(text)            
            if m2:
                output = []
                idx = 0
                # reduce URL data and add communities flag
                for item in m2:
                    username = self.__decodeToUtf8(item[1])
                    if self.m_scanMutualFriendsOnly:
                        if username in m1:                            
                            if item[0] == 'C': output.append((True, username))
                            else: output.append((False, username))
                        else:
                            self.__DDD("Skipping user " + str(username) + " (mutual friends only)")
                    else:
                            if item[0] == 'C': output.append((True, username))
                            else: output.append((False, username))
                    idx += 1
                return output
        return None
    
    def __fullFetch(self, comm, friendName, curLevel):
        depth = self.m_friendsTree.getNodeData(friendName, 'D')        
        if self.m_friendsTree.hasNode(friendName) and len(self.m_friendsTree.getRefSet(friendName)) != 0 and \
        self.m_friendsTree.getMaxDepth() - 2 != depth:             
            self.__DDD("Using cache for " + friendName)
            tmp = self.m_friendsTree.getRefSet(friendName)
            friendsOfFriend = [0] * len(tmp)
            idx = 0            
            for item in tmp:
                flag = self.m_friendsTree.getNodeData(item, 'C')
                friendsOfFriend[idx] = (flag, item)
                idx += 1
            self.m_stat.m_totalCacheUsed += 1
        else:
            rawData = self.__getUserFriends(comm, friendName)
            if rawData == None:                    
                return None
            friendsOfFriend = self.__fetchUserFriendsFromData(rawData)
            
        return friendsOfFriend
    
    def __fetchInterests(self, username):
        # get user interests            
        if self.m_collectInterests and self.m_friendsTree.getNodeData(username, 'I') == None:
            # get user interests if needed
            self.__DDD("Getting interests data for " + str(username))
            rawData = self.__getUserInterests(username)
            interestsData = self.__fetchUserInterestsFromData(rawData)
            self.m_friendsTree.addNodeData(username, 'I', interestsData)
            
    def __updateProgress(self):
        self.m_stat.updateProgress()
        if self.m_updateCallback != None:
            self.m_updateCallback(self.m_stat)                    

    def __traverseFriends(self, friendsList, history, currUser, checkedUsers, curLevel, maxLevel):
        # main recursive method which collects friends hierarchy and saves
        # it as a hashtree data structure
        
        curUserIdx = 0
        totalUsers = len(friendsList)
        
        if curLevel == maxLevel:
            return                        
        
        self.__fetchInterests(currUser)
        
        for friendData in friendsList:
                        
            friendName = friendData[1]
            
            communityFlag = False
            if friendData[0]: communityFlag = True
            
            # update progress
            if curLevel == 1:
                self.m_stat.m_percent = (100 * curUserIdx) / totalUsers
                self.__updateProgress()
                
            curUserIdx += 1
            
            # check for cycles
            if friendName in checkedUsers or currUser == friendName:
                # don't scan myself :)                
                self.__addAndLink(currUser, friendName, communityFlag)
                self.__setNodeDepth(friendName, curLevel)
                self.__fetchInterests(friendName)
                self.__DDD("Skipping " + friendName)                
                continue        
            
            if self.m_skipCommunities:
                # don't scan communities because many of them
                # have too much participants and it causes much more time
                # consumptions to perform a full scan                
                if communityFlag:                    
                    self.__DDD("Skipping community " + friendName)                                    
                    self.__addAndLink(currUser, friendName, communityFlag)
                    self.__setNodeDepth(friendName, curLevel)                    
                    self.m_stat.m_commSkipped += 1                    
                    continue                                                
            
            if curLevel == maxLevel - 1:
                # if the next level will be the maximum recursion level
                # don't load the user profile                
                self.m_stat.m_totalUsersChecked += 1
                self.__addAndLink(currUser, friendName, communityFlag)
                self.__setNodeDepth(friendName, curLevel)
                self.__fetchInterests(friendName)
                continue
            
            friendsOfFriend = self.__fullFetch(communityFlag, friendName, curLevel)
 
            self.__DDD("Checking friend " + friendName)
            self.m_stat.m_totalUsersChecked += 1 
            # debug info
            temp = history[:]
            temp.append(friendName)
            self.__DDD("History: " + str(temp))                

            checkedUsers.add(currUser)

            #time.sleep(0)

            if friendsOfFriend != None:
                self.__addAndLink(currUser, friendName, communityFlag)
                self.__setNodeDepth(friendName, curLevel)
                self.__traverseFriends(friendsOfFriend, temp, friendName, copy.deepcopy(checkedUsers), curLevel + 1, maxLevel)
                
        # update progress to 100%
        if curLevel == 1:
            self.m_stat.m_percent = 100    
            self.__updateProgress()        

    def __DDD(self, val):
        # my own print method, it does nothing
        # but checks if there is a m_verbose flag enabled or not
        if self.m_verbose:
            print val
    
    def setVerbose(self, val):
        self.m_verbose = val
    
    def setSkipCommunities(self, val):
        self.m_skipCommunities = val
    
    def setCollectInterests(self, val):
        self.m_collectInterests = val

    def setScanMutualFriendsOnly(self, val):
        self.m_scanMutualFriendsOnly = val
    
    def setUpdateCallback(self, callback):
        self.m_updateCallback = callback    
    
    def scanAllFriends(self, maxRecLevel):
        
        startTime = time.time()            
        
        # initial fetch
        rawData = self.__getUserFriends()
        friendsData = self.__fetchUserFriendsFromData(rawData)
                        
        # prepare initial friends tree
        self.m_friendsTree = HashTree()
        self.m_friendsTree.addNode(self.m_userName)
        self.m_friendsTree.addNodeData(self.m_userName, 'ROOT', 'True')
        self.m_friendsTree.setMaxDepth(maxRecLevel + 2)
                
        self.__setNodeDepth(self.m_userName, 0)
    
        if self.m_collectInterests:
            # get user interests if needed
            rawData = self.__getUserInterests(self.m_userName)
            interestsData = self.__fetchUserInterestsFromData(rawData)
            self.m_friendsTree.addNodeData(self.m_userName, 'I', interestsData)
        
        for user in friendsData:
            # add user
            self.m_friendsTree.addNode(user[1])
            self.m_friendsTree.setRef(self.m_userName, user[1])
            # add URL
            self.m_friendsTree.addNodeData(user[1], 'C', user[0])
            # setup depth
            self.__setNodeDepth(user[1], 1)
                        
        self.__traverseFriends(friendsData, [self.m_userName], self.m_userName, set([self.m_userName]), 1, maxRecLevel + 1)       
        
        timeTaken = time.time() - startTime

        self.m_stat.m_timeTaken = timeTaken        
        return self.m_friendsTree    
    
    def performRPC(self, method, useCookie = True):
        chal = self.__getChalResponse()
        # construct request
        dataDict = {
                    "mode": method,
                    "user": self.m_userName,
                    "includefriendof": 1,
                    #"auth_method": "cookie",
                    #"chal": chal['challenge']
                    "password": self.m_password
                   }
                
        # encode it to send by means of POST
        data = urllib.urlencode(dataDict)
        # create opener
        request = urllib2.Request(self.flatServerUrl, data)
        
        if (useCookie):
            cookieHandler = urllib2.HTTPCookieProcessor()
            opener = urllib2.build_opener(cookieHandler)
            fp = opener.open(request)
        else:
            fp = urllib2.urlopen(request, timeout = LJUserSession.TIMEOUT_SEC)
                    
        result = fp.read()
        fp.close()
        
        return result

    def loginLJAccount(self):
        # tries to login to livejournal with the given
        # username and password
        response = self.__getChalResponse()
        auth_response = self.__getAuthResponse(self.m_password, response['challenge'])
        
        authData = urllib.urlencode({
        				"chal": response['challenge'],
        				"response": auth_response,
        				"user": self.m_userName
        				})
        
        cookieHandler = urllib2.HTTPCookieProcessor()
        opener = urllib2.build_opener(cookieHandler)                
        request = urllib2.Request(self.authServerUrl, authData)        
        # get cookie
        fp = opener.open(request)
        fp.close()
        
        # check if authorization is ok        
        if len(cookieHandler.cookiejar) < 2:
        	raise AuthError
                
        return cookieHandler.cookiejar
    
    def getStatDataStruct(self):
        return self.m_stat

    @staticmethod
    def checkUserExists(username):
        # checks if entered user has livejournal account and we can connect to it
        url = 'http://www.livejournal.com/misc/fdata.bml?comm=0&user=' + username        
        request = urllib2.Request(url)
        try:
            fp = urllib2.urlopen(request, timeout = LJUserSession.TIMEOUT_SEC)
            text = fp.read()
            fp.close()
            m1 = LJUserSession.reInvalidUser.search(text)
            return m1 == None
        except:
            return False
        return True
          

class StatDataStruct:
    
    def __init__(self):
        self.m_totalUsersChecked = 0
        self.m_totalCacheUsed = 0
        self.m_commSkipped = 0
        self.m_timeTaken = 0
        self.m_totalNetErrSkip = 0
        self.m_percent = 0
        self.m_lastStr = ""
    
    def updateProgress(self):
        b = '\b' * (len(self.m_lastStr) + 1)
        self.m_lastStr = str(self.m_percent) + "%"
        print b + self.m_lastStr,        
    
    def showStats(self):                
        print "Users checked: " + str(self.m_totalUsersChecked)
        print "Communities skipped: " + str(self.m_commSkipped)
        print "Cache entries used: " + str(self.m_totalCacheUsed)
        print "Skipped due to network errors: " + str(self.m_totalNetErrSkip)
        print "Time taken: " + str(int(self.m_timeTaken) / 3600) + " hours, " + str(int(self.m_timeTaken) / 60) + " minutes, " + str(int(self.m_timeTaken) % 60) + " seconds"
